import "dotenv/config";
import { HumanMessage } from "@langchain/core/messages";
import { ChatOpenAI } from "@langchain/openai";
import { Document } from "langchain/document";
import { TextLoader } from "langchain/document_loaders/fs/text";
// import { PDFLoader } from "langchain/document_loaders/fs/pdf";
import { PDFLoader } from "@langchain/community/document_loaders/fs/pdf";
import { DirectoryLoader } from "langchain/document_loaders/fs/directory";
import { GithubRepoLoader } from "@langchain/community/document_loaders/web/github";
import { CheerioWebBaseLoader } from "@langchain/community/document_loaders/web/cheerio";
import { SerpAPILoader } from "@langchain/community/document_loaders/web/serpapi";

// import * as pdfParse from "pdf-parse";

const test = new Document({
  id: "test",
  pageContent: "test text",
  metadata: { source: "ABC Title" },
});

// console.log(test.metadata); // "test text"

const textLoader = new TextLoader("./data/test.txt");
const text = await textLoader.load();
// console.log(text);

const pageContent = text.map((t) => new HumanMessage(t.pageContent));
// console.log(pageContent); // ["test text"]

const pdfLoader = new PDFLoader("./data/test.pdf", { splitPages: false });
const pdf = await pdfLoader.load();
// console.log("pdf", pdf[0].metadata);

const directoryLoader = new DirectoryLoader("./data", {
  ".pdf": (path) => new PDFLoader(path, { splitPages: false }),
  ".txt": (path) => new TextLoader(path),
});

const docs = await directoryLoader.load();
// console.log(docs); // [{}, {}]

const loader = new GithubRepoLoader(
  "https://github.com/RealKai42/qwerty-learner",
  {
    branch: "master",
    recursive: false,
    unknown: "warn",
    ignorePaths: ["*.md", "yarn.lock", "*.json"],
    accessToken: "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",
  }
);

// const re = await loader.load();
// console.log(re);

const cheerioLoader = new CheerioWebBaseLoader(
  "https://kaiyi.cool/blog/github-copilot",
  {
    selector: "h3",
  }
);

// const cheerio = await cheerioLoader.load();
// console.log(cheerio[0].pageContent);

const serpapiLoader = new SerpAPILoader({
  q: "什么是 github copliot",
  apiKey: process.env.SERPAPI_API_KEY,
});

const serpapi = await serpapiLoader.load();
console.log(serpapi);

// const chatModel = new ChatOpenAI({
//   configuration: {
//     baseURL: process.env.OPENAI_API_BASE,
//     apiKey: process.env.OPENAI_API_KEY,
//   },
// });

// const r = await chatModel.invoke(pageContent);
// console.log(r.content); // ["test text"]
